// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved.

// Package recover a series of service function
package recover

import (
	"context"
	"errors"
	"testing"
	"time"

	"github.com/agiledragon/gomonkey/v2"
	"github.com/smartystreets/goconvey/convey"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"ascend-common/api"
	"clusterd/pkg/common/constant"
	"clusterd/pkg/domain/common"
	"clusterd/pkg/domain/pod"
	"clusterd/pkg/interface/grpc/recover"
	"clusterd/pkg/interface/kube"
)

type hotSwitchTestCase struct {
	name                  string
	controller            *EventController
	mockFaultRanks        []*pb.FaultRank
	mockSignalEnqueueResp string
	mockSignalEnqueueCode common.RespCode
	mockSignalEnqueueErr  error
	expectedEvent         string
	expectedCode          common.RespCode
	expectedError         error
}

type signalTestConfig struct {
	signalType string
	actions    []string
	strategy   string
}

func buildHotSwitchTestCases() []hotSwitchTestCase {
	return []hotSwitchTestCase{
		{
			name:                  "successful hot switch preparation",
			controller:            &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-1"}, uuid: "test-uuid-1"},
			mockFaultRanks:        []*pb.FaultRank{{RankId: "0"}},
			mockSignalEnqueueResp: "success",
			mockSignalEnqueueCode: common.OK,
			mockSignalEnqueueErr:  nil,
			expectedEvent:         "success",
			expectedCode:          common.OK,
			expectedError:         nil,
		},
		{
			name:                  "hot switch preparation with error",
			controller:            &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-2"}, uuid: "test-uuid-2"},
			mockFaultRanks:        []*pb.FaultRank{{RankId: "1"}},
			mockSignalEnqueueResp: "",
			mockSignalEnqueueCode: common.ServerInnerError,
			mockSignalEnqueueErr:  nil,
			expectedEvent:         "",
			expectedCode:          common.ServerInnerError,
			expectedError:         nil,
		},
	}
}

func buildSignalTestCases(signalType string, actions []string, strategy string) []hotSwitchTestCase {
	return []hotSwitchTestCase{
		{
			name:                  "successful " + signalType + " handling",
			controller:            &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-1"}, uuid: "test-uuid-1"},
			mockSignalEnqueueResp: "success",
			mockSignalEnqueueCode: common.OK,
			mockSignalEnqueueErr:  nil,
			expectedEvent:         "success",
			expectedCode:          common.OK,
			expectedError:         nil,
		},
		{
			name:                  signalType + " handling with error",
			controller:            &EventController{jobInfo: common.JobBaseInfo{JobId: "test-job-2"}, uuid: "test-uuid-2"},
			mockSignalEnqueueResp: "",
			mockSignalEnqueueCode: common.ServerInnerError,
			mockSignalEnqueueErr:  nil,
			expectedEvent:         "",
			expectedCode:          common.ServerInnerError,
			expectedError:         nil,
		},
	}
}

func runSignalTest(t *testing.T, tests []hotSwitchTestCase, testFunc func(*EventController) (string, common.RespCode, error), config signalTestConfig) {
	for _, tt := range tests {
		convey.Convey(tt.name, t, func() {
			patch := gomonkey.ApplyPrivateMethod((*EventController)(nil), "signalEnqueue",
				func(_ *EventController, signal *pb.ProcessManageSignal) (string, common.RespCode, error) {
					convey.So(signal.Uuid, convey.ShouldEqual, tt.controller.uuid)
					convey.So(signal.JobId, convey.ShouldEqual, tt.controller.jobInfo.JobId)
					convey.So(signal.SignalType, convey.ShouldEqual, config.signalType)
					convey.So(signal.Actions, convey.ShouldResemble, config.actions)
					convey.So(signal.ChangeStrategy, convey.ShouldEqual, config.strategy)
					return tt.mockSignalEnqueueResp, tt.mockSignalEnqueueCode, tt.mockSignalEnqueueErr
				})
			defer patch.Reset()
			event, code, err := testFunc(tt.controller)
			convey.So(event, convey.ShouldEqual, tt.expectedEvent)
			convey.So(code, convey.ShouldEqual, tt.expectedCode)
			convey.So(err, convey.ShouldEqual, tt.expectedError)
		})
	}
}

func TestNotifyPrepareHotSwitch(t *testing.T) {
	tests := buildHotSwitchTestCases()
	for _, tt := range tests {
		convey.Convey(tt.name, t, func() {
			patch := gomonkey.NewPatches()
			defer patch.Reset()
			patch.ApplyPrivateMethod(tt.controller, "normalFaultAssociateSameNodeRank",
				func(_ *EventController) ([]*pb.FaultRank, map[string]string) {
					return tt.mockFaultRanks, nil
				}).ApplyPrivateMethod(tt.controller, "signalEnqueue",
				func(_ *EventController, signal *pb.ProcessManageSignal) (string, common.RespCode, error) {
					convey.So(signal.Uuid, convey.ShouldEqual, tt.controller.uuid)
					convey.So(signal.JobId, convey.ShouldEqual, tt.controller.jobInfo.JobId)
					convey.So(signal.SignalType, convey.ShouldEqual, constant.HotSwitchSignalType)
					convey.So(signal.Actions, convey.ShouldResemble, hotSwitchActions)
					convey.So(signal.FaultRanks, convey.ShouldResemble, tt.mockFaultRanks)
					convey.So(signal.ChangeStrategy, convey.ShouldEqual, "")
					return tt.mockSignalEnqueueResp, tt.mockSignalEnqueueCode, tt.mockSignalEnqueueErr
				})

			event, code, err := tt.controller.notifyPrepareHotSwitch()

			convey.So(event, convey.ShouldEqual, tt.expectedEvent)
			convey.So(code, convey.ShouldEqual, tt.expectedCode)
			convey.So(err, convey.ShouldEqual, tt.expectedError)
		})
	}
}

func TestNotifyNewPodFailedHandler(t *testing.T) {
	tests := buildSignalTestCases("new pod failed", stopHotSwitchActions, "")
	config := signalTestConfig{
		signalType: constant.HotSwitchSignalType,
		actions:    stopHotSwitchActions,
		strategy:   "",
	}
	testFunc := func(controller *EventController) (string, common.RespCode, error) {
		return controller.notifyNewPodFailedHandler()
	}
	runSignalTest(t, tests, testFunc, config)
}

func TestNotifyNewPodRunningHandler(t *testing.T) {
	tests := buildSignalTestCases("new pod running", newPodRunningActions, "")
	config := signalTestConfig{
		signalType: constant.HotSwitchSignalType,
		actions:    newPodRunningActions,
		strategy:   "",
	}
	testFunc := func(controller *EventController) (string, common.RespCode, error) {
		return controller.notifyNewPodRunningHandler()
	}
	runSignalTest(t, tests, testFunc, config)
}

func TestNotifyRestartTrain(t *testing.T) {
	tests := buildSignalTestCases("restart train", changeStrategyActions, constant.ProcessMigration)
	config := signalTestConfig{
		signalType: constant.ChangeStrategySignalType,
		actions:    changeStrategyActions,
		strategy:   constant.ProcessMigration,
	}
	testFunc := func(controller *EventController) (string, common.RespCode, error) {
		return controller.notifyRestartTrain()
	}
	runSignalTest(t, tests, testFunc, config)
}

type hotSwitchTestCaseB struct {
	name            string
	podName         string
	podExist        bool
	annotationExist bool
	deleteErr       error
	newPodExist     bool
	patchErr        error
	expectedCode    common.RespCode
	expectError     bool
	expectedEvent   string
	labelErr        error
	faultPods       map[string]string
}

func TestCleanStateWhenFailed(t *testing.T) {
	ctl := buildBaseController()
	testCases := buildHotSwitchTestCase1()
	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			doPatches(patches, ctl, tc)
			event, code, err := ctl.cleanStateWhenFailed()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, "")
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
		})
	}
}

func doPatches(patches *gomonkey.Patches, ctl *EventController, tc hotSwitchTestCaseB) {
	patchReset(patches, ctl)
	if !tc.podExist {
		patchGetPodByPodId(patches, v1.Pod{}, false)
	} else {
		testPod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fault-pod", Namespace: "default"}}
		if tc.annotationExist {
			testPod.Annotations = map[string]string{api.InHotSwitchFlowKey: "true", api.BackupNewPodNameKey: "new-pod"}
		} else {
			testPod.Annotations = make(map[string]string)
		}
		patchGetPodByPodId(patches, testPod, true)
		patches.ApplyFuncReturn(kube.DeletePodAnnotation, tc.deleteErr)
		if tc.newPodExist {
			patches.ApplyFuncReturn(pod.GetPodByJobIdAndPodName,
				v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new-pod", Namespace: "default"}}, true)
		} else {
			patches.ApplyFuncReturn(pod.GetPodByJobIdAndPodName, v1.Pod{}, false)
		}
		patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, tc.patchErr)
	}
}

func patchGetPodByPodId(patches *gomonkey.Patches, p v1.Pod, exist bool) *gomonkey.Patches {
	return patches.ApplyFuncReturn(pod.GetPodByPodId, p, exist)
}

func buildBaseController() *EventController {
	ctl := &EventController{
		currentHotSwitchFaultPodId: "test-pod-id",
		jobInfo:                    common.JobBaseInfo{JobId: "test-job-id"},
	}
	return ctl
}

func buildHotSwitchTestCase1() []hotSwitchTestCaseB {
	testCases := []hotSwitchTestCaseB{
		{name: "pod not exist",
			podExist:     false,
			expectedCode: common.OK,
			expectError:  false},
		{name: "annotation not exist",
			podExist:        true,
			annotationExist: false,
			expectedCode:    common.OK,
			expectError:     false},
		{name: "delete annotation failed",
			podExist:        true,
			annotationExist: true,
			deleteErr:       errors.New("delete failed"),
			expectedCode:    common.ServerInnerError,
			expectError:     true},
		{name: "new pod not exist",
			podExist:        true,
			annotationExist: true,
			deleteErr:       nil,
			newPodExist:     false,
			expectedCode:    common.ServerInnerError,
			expectError:     true},
		{name: "patch new pod failed",
			podExist:        true,
			annotationExist: true,
			deleteErr:       nil,
			newPodExist:     true,
			patchErr:        errors.New("patch failed"),
			expectedCode:    common.ServerInnerError,
			expectError:     true},
		{name: "success",
			podExist:        true,
			annotationExist: true,
			deleteErr:       nil,
			newPodExist:     true,
			patchErr:        nil,
			expectedCode:    common.OK,
			expectError:     false},
	}
	return testCases
}

func TestCleanStateWhenSuccess(t *testing.T) {
	ctl := &EventController{
		currentHotSwitchBackupPodId: "test-backup-pod-id",
		jobInfo:                     common.JobBaseInfo{JobId: "test-job-id"},
	}
	testCases := buildHotSwitchTestCases2()
	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			patchReset(patches, ctl)
			if !tc.podExist {
				patchGetPodByPodId(patches, v1.Pod{}, false)
			} else {
				testPod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "backup-pod", Namespace: "default"}}
				patchGetPodByPodId(patches, testPod, true)
				patches.ApplyFuncReturn(kube.DeletePodAnnotation, tc.deleteErr)
			}
			event, code, err := ctl.cleanStateWhenSuccess()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, "")
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
		})
	}
}

func buildHotSwitchTestCases2() []hotSwitchTestCaseB {
	testCases := []hotSwitchTestCaseB{
		{
			name:         "pod not exist",
			podExist:     false,
			expectedCode: common.OK,
			expectError:  false,
		}, {
			name:         "delete annotation failed",
			podExist:     true,
			deleteErr:    errors.New("delete failed"),
			expectedCode: common.ServerInnerError,
			expectError:  true,
		}, {
			name:         "success",
			podExist:     true,
			deleteErr:    nil,
			expectedCode: common.OK,
			expectError:  false,
		},
	}
	return testCases
}

func TestNotifyStopJob(t *testing.T) {
	ctl := &EventController{
		jobInfo: common.JobBaseInfo{
			JobId:   "test-job-id",
			JobName: "test-job-name",
		},
	}

	testCases := []hotSwitchTestCaseB{
		{
			name:          "pod not found",
			podName:       "",
			expectError:   false,
			expectedCode:  common.OK,
			expectedEvent: common.NotifySuccessEvent,
		},
		{
			name:          "pod found and stop job",
			podName:       "test-pod",
			expectError:   false,
			expectedCode:  common.OK,
			expectedEvent: common.NotifySuccessEvent,
		},
	}

	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			patchReset(patches, ctl)
			patches.ApplyFunc(pod.GetPodByRankIndex, func(jobId string, rankIndex string) v1.Pod {
				return v1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      tc.podName,
						Namespace: "default",
					},
				}
			})
			patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, nil)
			patches.ApplyFuncReturn(kube.RetryPatchPodLabels, nil)
			event, code, err := ctl.notifyStopJob()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, tc.expectedEvent)
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
		})
	}
}

func patchReset(patches *gomonkey.Patches, ctl *EventController) *gomonkey.Patches {
	return patches.ApplyPrivateMethod(ctl, "reset", func(bool) {})
}

func TestNotifyDeleteOldPod(t *testing.T) {
	controller := &EventController{
		currentHotSwitchFaultPodId: "test-pod-id",
		jobInfo:                    common.JobBaseInfo{JobId: "test-job-id"},
	}
	testCases := buildHotSwitchTestCase3()

	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			patchReset(patches, controller)
			if !tc.podExist {
				patchGetPodByPodId(patches, v1.Pod{}, false)
			} else {
				testPod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}}
				patchGetPodByPodId(patches, testPod, true)
				patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, tc.patchErr)
				patches.ApplyFuncReturn(kube.RetryPatchPodLabels, tc.labelErr)
			}
			event, code, err := controller.notifyDeleteOldPod()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, "")
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
		})
	}
}

func buildHotSwitchTestCase3() []hotSwitchTestCaseB {
	testCases := []hotSwitchTestCaseB{
		{
			name:         "pod not exist",
			podExist:     false,
			expectedCode: common.OK,
			expectError:  false,
		}, {
			name:         "patch annotation failed",
			podExist:     true,
			patchErr:     errors.New("patch annotation failed"),
			expectedCode: common.OK,
			expectError:  false,
		}, {
			name:         "patch label failed",
			podExist:     true,
			labelErr:     errors.New("patch label failed"),
			expectedCode: common.OK,
			expectError:  false,
		}, {
			name:         "success",
			podExist:     true,
			patchErr:     nil,
			labelErr:     nil,
			expectedCode: common.OK,
			expectError:  false},
	}
	return testCases
}

func TestNotifyCreateNewPod(t *testing.T) {
	controller := &EventController{
		jobInfo: common.JobBaseInfo{JobId: "test-job-id"},
	}
	testCases := buildHotSwitchTestCases3()
	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			patches.ApplyMethodReturn(controller, "GetFaultPod", tc.faultPods)
			if len(tc.faultPods) == 0 {
				// do nothing
			} else if !tc.podExist {
				patchGetPodByPodId(patches, v1.Pod{}, false)
			} else {
				testPod := v1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-pod",
						Namespace: "default",
						UID:       "test-pod-uid",
					},
				}
				patchGetPodByPodId(patches, testPod, true)
				patches.ApplyFuncReturn(kube.RetryPatchPodAnnotations, tc.patchErr)
			}
			event, code, err := controller.notifyCreateNewPod()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, "")
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
			if tc.patchErr == nil && len(tc.faultPods) > 0 && tc.podExist {
				convey.So(controller.currentHotSwitchFaultPodId, convey.ShouldEqual, "test-pod-uid")
			}
		})
	}
}

func buildHotSwitchTestCases3() []hotSwitchTestCaseB {
	testCases := []hotSwitchTestCaseB{
		{
			name:         "should return OK when no fault pods",
			faultPods:    map[string]string{},
			expectedCode: common.OK,
			expectError:  false,
		},
		{
			name:         "should return OK when pod not exist",
			faultPods:    map[string]string{"0": "test-pod-id"},
			podExist:     false,
			expectedCode: common.OK,
			expectError:  false,
		},
		{
			name:         "should return OK when patch annotation failed",
			faultPods:    map[string]string{"0": "test-pod-id"},
			podExist:     true,
			patchErr:     errors.New("patch failed"),
			expectedCode: common.OK,
			expectError:  false,
		},
		{
			name:         "should return OK when success",
			faultPods:    map[string]string{"0": "test-pod-id"},
			podExist:     true,
			patchErr:     nil,
			expectedCode: common.OK,
			expectError:  false,
		},
	}
	return testCases
}

type hotSwitchTestCaseC struct {
	name             string
	chanNil          bool
	channelClosed    bool
	contextCanceled  bool
	timeout          bool
	strategies       []string
	expectedCode     common.RespCode
	expectedCtlEvent string
	expectedRtnEvent string
	expectError      bool
	responseCode     int32
	podStatus        v1.PodPhase
}

func TestWaitReportPauseTrainResult(t *testing.T) {
	controller := &EventController{uuid: "test-uuid", jobInfo: common.JobBaseInfo{JobId: "test-job-id"}}
	testCases := buildHotSwitchTestCases4()
	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			strategyChan := make(chan *pb.RecoverStrategyRequest, 1)
			ctx, cancel := context.WithCancel(context.Background())
			if tc.chanNil {
				patchGetCtxAndReportRecoverStrategyChan(patches, controller, ctx, nil)
			} else {
				patchGetCtxAndReportRecoverStrategyChan(patches, controller, ctx, strategyChan)
			}
			if tc.channelClosed {
				close(strategyChan)
			} else if tc.contextCanceled {
				cancel()
			} else if tc.timeout {
				patchTimeOut(patches)
			} else {
				safeWriteToChannel(t, strategyChan, &pb.RecoverStrategyRequest{Strategies: tc.strategies})
			}
			eventAdded := ""
			patches.ApplyPrivateMethod(controller, "addEvent", func(_ *EventController, event string) {
				eventAdded = event
			})
			event, code, err := controller.waitReportPauseTrainResult()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, tc.expectedRtnEvent)
			convey.So(eventAdded, convey.ShouldEqual, tc.expectedCtlEvent)
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
			if !tc.contextCanceled && ctx.Err() == nil {
				cancel()
			}
			if !tc.channelClosed {
				close(strategyChan)
			}
		})
	}
}

func patchTimeOut(patches *gomonkey.Patches) *gomonkey.Patches {
	return patches.ApplyFunc(time.After, func(d time.Duration) <-chan time.Time {
		ch := make(chan time.Time, 1)
		ch <- time.Time{}
		return ch
	})
}

func patchGetCtxAndReportRecoverStrategyChan(patches *gomonkey.Patches, controller *EventController,
	ctx context.Context, strategyChan chan *pb.RecoverStrategyRequest) *gomonkey.Patches {
	return patches.ApplyPrivateMethod(controller, "getCtxAndReportRecoverStrategyChan",
		func(*EventController) (context.Context, chan *pb.RecoverStrategyRequest) {
			return ctx, strategyChan
		})
}

func buildHotSwitchTestCases4() []hotSwitchTestCaseC {
	testCases := []hotSwitchTestCaseC{
		{name: "should return server inner error when strategy channel is nil",
			chanNil:      true,
			expectedCode: common.ServerInnerError,
			expectError:  true,
		}, {
			name:          "should return OK when channel closed",
			channelClosed: true,
			expectedCode:  common.OK,
			expectError:   false,
		}, {
			name:            "should return controller event cancel when context canceled",
			contextCanceled: true,
			expectedCode:    common.ControllerEventCancel,
			expectError:     false,
		}, {
			name:             "should return wait report timeout when timeout occurs",
			timeout:          true,
			expectedCode:     common.WaitReportTimeout,
			expectedRtnEvent: common.ReportTimeoutEvent,
			expectError:      false,
		}, {
			name:             "should add exit event when strategies is empty",
			strategies:       []string{},
			expectedCode:     common.OK,
			expectedCtlEvent: common.ExitEvent,
			expectedRtnEvent: "",
			expectError:      false,
		}, {
			name:             "should add exit event when exit strategy is received",
			strategies:       []string{constant.ProcessExitStrategyName},
			expectedCode:     common.OK,
			expectedCtlEvent: common.ExitEvent,
			expectedRtnEvent: "",
			expectError:      false,
		}, {
			name:             "should add migration event when migration strategy is received",
			strategies:       []string{"migration"},
			expectedCode:     common.OK,
			expectedCtlEvent: common.MigrationEvent,
			expectedRtnEvent: "",
			expectError:      false,
		},
	}
	return testCases
}

func safeWriteToChannel[T any](t *testing.T, ch chan T, value T) {
	select {
	case ch <- value:
		t.Logf("write to channel success")
	default:
		t.Errorf("write to channel failed: channel is full or nil")
	}
}

func TestHandleWaitReportRestartTrainStatus(t *testing.T) {
	controller := &EventController{uuid: "test-uuid", jobInfo: common.JobBaseInfo{JobId: "test-job-id"}}
	testCases := buildHotSwitchTestCases5()
	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()

			reportChan := make(chan *pb.RecoverStatusRequest, 1)
			ctx, cancel := context.WithCancel(context.Background())
			if tc.chanNil {
				patchGetCtxAndResultChan(patches, controller, ctx, nil)
			} else {
				patchGetCtxAndResultChan(patches, controller, ctx, reportChan)
			}
			if tc.contextCanceled {
				cancel()
			} else if tc.timeout {
				patchTimeOut(patches)
			} else {
				reportChan <- &pb.RecoverStatusRequest{Status: &pb.Status{Code: tc.responseCode}}
			}
			event, code, err := controller.handleWaitReportRestartTrainStatus()
			convey.So(code, convey.ShouldEqual, tc.expectedCode)
			convey.So(event, convey.ShouldEqual, tc.expectedRtnEvent)
			if tc.expectError {
				convey.So(err, convey.ShouldNotBeNil)
			} else {
				convey.So(err, convey.ShouldBeNil)
			}
			if !tc.contextCanceled && ctx.Err() == nil {
				cancel()
			}
			close(reportChan)
		})
	}
}

func buildHotSwitchTestCases5() []hotSwitchTestCaseC {
	testCases := []hotSwitchTestCaseC{
		{
			name:             "should return OK and restart failed event when report channel is nil",
			chanNil:          true,
			expectedCode:     common.OK,
			expectedRtnEvent: common.RestartFaildEvent,
			expectError:      false,
		},
		{
			name:            "should return controller event cancel when context canceled",
			contextCanceled: true,
			expectedCode:    common.ControllerEventCancel,
			expectError:     false,
		},
		{
			name:             "should return wait report timeout when timeout occurs",
			timeout:          true,
			expectedCode:     common.WaitReportTimeout,
			expectedRtnEvent: common.ReportTimeoutEvent,
			expectError:      false,
		},
		{
			name:             "should return OK and restart failed event when unrecoverable retry error received",
			responseCode:     int32(common.UnRecoverableRetryError),
			expectedCode:     common.OK,
			expectedRtnEvent: common.RestartFaildEvent,
			expectError:      false,
		},
		{
			name:             "should return OK and restart success event when success response received",
			responseCode:     int32(common.OK),
			expectedCode:     common.OK,
			expectedRtnEvent: common.RestartSuccessEvent,
			expectError:      false,
		},
		{
			name:             "should return OK when unexpected response code received",
			responseCode:     int32(999),
			expectedCode:     common.OK,
			expectedRtnEvent: "",
			expectError:      false,
		},
	}
	return testCases
}

func patchGetCtxAndResultChan(patches *gomonkey.Patches, controller *EventController, ctx context.Context, reportChan chan *pb.RecoverStatusRequest) *gomonkey.Patches {
	return patches.ApplyPrivateMethod(controller, "getCtxAndResultChan", func(*EventController) (context.Context, chan *pb.RecoverStatusRequest) {
		return ctx, reportChan
	})
}

func patchGetCtxAndNewStatusMonitorChan(patches *gomonkey.Patches, controller *EventController, ctx context.Context, podPhase chan v1.PodPhase) *gomonkey.Patches {
	return patches.ApplyPrivateMethod(controller, "getCtxAndNewStatusMonitorChan", func(*EventController) (context.Context, chan v1.PodPhase) {
		return ctx, podPhase
	})
}

func TestMonitorNewPodStatus(t *testing.T) {
	controller := &EventController{uuid: "test-uuid", jobInfo: common.JobBaseInfo{JobId: "test-job-id"}}
	testCases := buildHotSwitchTestCases6()
	for _, tc := range testCases {
		convey.Convey(tc.name, t, func() {
			patches := gomonkey.NewPatches()
			defer patches.Reset()
			statusChan := make(chan v1.PodPhase, 1)
			ctx, cancel := context.WithCancel(context.Background())
			eventAdded := ""
			patches.ApplyPrivateMethod(controller, "addEvent", func(_ *EventController, event string) {
				eventAdded = event
			})
			if tc.chanNil {
				patchGetCtxAndNewStatusMonitorChan(patches, controller, ctx, nil)
			} else {
				patchGetCtxAndNewStatusMonitorChan(patches, controller, ctx, statusChan)
			}

			if tc.channelClosed {
				close(statusChan)
			} else if tc.contextCanceled {
				cancel()
			} else if tc.timeout {
				patchTimeOut(patches)
			} else if tc.podStatus != "" {
				safeWriteToChannel(t, statusChan, tc.podStatus)
			}
			monitorNewPodStatus(controller)
			if tc.expectedCtlEvent != "" {
				convey.So(eventAdded, convey.ShouldEqual, tc.expectedCtlEvent)
			}
			if !tc.contextCanceled && ctx.Err() == nil {
				cancel()
			}
			if !tc.channelClosed && !tc.chanNil {
				close(statusChan)
			}
		})
	}
}

func buildHotSwitchTestCases6() []hotSwitchTestCaseC {
	testCases := []hotSwitchTestCaseC{
		{
			name:    "should not panic when channel is nil",
			chanNil: true,
		},
		{
			name:          "should not panic when channel closed",
			channelClosed: true,
		},
		{
			name:            "should not panic when context canceled",
			contextCanceled: true,
		},
		{
			name:             "should add new pod timeout event when timeout occurs",
			timeout:          true,
			expectedCtlEvent: common.NewPodTimeoutEvent,
		},
		{
			name:             "should add new pod running event when pod running success",
			podStatus:        v1.PodRunning,
			expectedCtlEvent: common.NewPodRunningEvent,
		},
		{
			name:             "should add new pod running event when pod in other status",
			podStatus:        v1.PodPending,
			expectedCtlEvent: common.NewPodRunningEvent,
		},
	}
	return testCases
}
